diff --git a/swh/web/misc/coverage.py b/swh/web/misc/coverage.py
--- a/swh/web/misc/coverage.py
+++ b/swh/web/misc/coverage.py
@@ -16,10 +16,10 @@
from swh.scheduler.model import SchedulerMetrics
from swh.web.common import archive
-from swh.web.common.origin_save import get_savable_visit_types
from swh.web.common.utils import (
django_cache,
get_deposits_list,
+ is_swh_web_development,
is_swh_web_production,
reverse,
)
@@ -334,7 +334,9 @@
def _swh_coverage(request: HttpRequest) -> HttpResponse:
use_cache = is_swh_web_production(request)
listers_metrics = _get_listers_metrics(use_cache)
+
for origins in listed_origins["origins"]:
+ origins["count"] = "0"
origins["instances"] = {}
origins_type = origins["type"]
@@ -343,14 +345,14 @@
if origins_type in ("nixos", "guix"):
count = _get_nixguix_origins_count(origins["search_pattern"], use_cache)
- origins["count"] = f"{count:,}" if count else ""
+ origins["count"] = f"{count:,}"
origins["instances"][origins_type] = {"nixguix": {"count": count}}
if origins_type not in listers_metrics:
continue
count_total = sum(
- [metrics.origins_known for _, metrics in listers_metrics[origins_type]]
+ [metrics.origins_enabled for _, metrics in listers_metrics[origins_type]]
)
count_never_visited = sum(
[
@@ -363,13 +365,11 @@
origins["count"] = f"{count:,}"
origins["instances"] = defaultdict(dict)
for instance, metrics in listers_metrics[origins_type]:
- # these types are available in staging/docker but not yet in production
- if (
- metrics.visit_type in ("bzr", "cvs")
- and metrics.visit_type not in get_savable_visit_types()
- ):
+ instance_count = metrics.origins_enabled - metrics.origins_never_visited
+ # no archived origins for that visit type, skip it
+ if instance_count == 0:
continue
- instance_count = metrics.origins_known - metrics.origins_never_visited
+
origins["instances"][instance].update(
{metrics.visit_type: {"count": f"{instance_count:,}"}}
)
@@ -396,6 +396,12 @@
search_url = _search_url(search_pattern, visit_type)
visit_types[visit_type]["search_url"] = search_url
+ # filter out origin types without archived origins on production and staging
+ if not is_swh_web_development(request):
+ listed_origins["origins"] = list(
+ filter(lambda o: o["count"] != "0", listed_origins["origins"])
+ )
+
for origins in legacy_origins["origins"]:
origins["search_urls"] = {}
for visit_type in origins["visit_types"]:
@@ -406,6 +412,7 @@
deposits_counts = _get_deposits_netloc_counts(use_cache)
for origins in deposited_origins["origins"]:
+ origins["count"] = "0"
if origins["search_pattern"] in deposits_counts:
origins["count"] = f"{deposits_counts[origins['search_pattern']]:,}"
origins["search_urls"] = {
diff --git a/swh/web/templates/misc/coverage.html b/swh/web/templates/misc/coverage.html
--- a/swh/web/templates/misc/coverage.html
+++ b/swh/web/templates/misc/coverage.html
@@ -102,7 +102,7 @@
{% for instance, visit_types in origins.instances.items %}
{% for visit_type, data in visit_types.items %}
{% if data.count %}
- <tr>
+ <tr id="{{ origins.type }}-{{ visit_type }}">
<td>{{ instance }}</td>
<td>{{ visit_type }}</td>
<td>{{ data.count }}</td>
@@ -126,7 +126,7 @@
</thead>
<tbody>
{% for visit_type, search_url in origins.search_urls.items %}
- <tr>
+ <tr id="{{ origins.type }}-{{ visit_type }}">
<td>{{ origins.type }}</td>
<td>{{ visit_type }}</td>
<td>
diff --git a/swh/web/tests/misc/test_coverage.py b/swh/web/tests/misc/test_coverage.py
--- a/swh/web/tests/misc/test_coverage.py
+++ b/swh/web/tests/misc/test_coverage.py
@@ -3,6 +3,7 @@
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import copy
from datetime import datetime, timezone
from itertools import chain
import os
@@ -18,7 +19,7 @@
from swh.web.common.utils import reverse
from swh.web.config import SWH_WEB_SERVER_NAME
from swh.web.misc.coverage import deposited_origins, legacy_origins, listed_origins
-from swh.web.tests.django_asserts import assert_contains
+from swh.web.tests.django_asserts import assert_contains, assert_not_contains
from swh.web.tests.utils import check_html_get_response, check_http_get_response
@@ -33,10 +34,10 @@
)
-visit_types = ["git", "hg", "svn", "bzr", "svn"]
+visit_types = ["git", "hg", "svn", "bzr", "cvs"]
-@pytest.fixture(autouse=True)
+@pytest.fixture
def archive_coverage_data(mocker, swh_scheduler):
"""Generate some sample scheduler metrics and some sample deposits
that will be consumed by the archive coverage view.
@@ -101,7 +102,8 @@
get_deposits_list.return_value = deposits
-def test_coverage_view_with_metrics(client):
+def test_coverage_view_with_metrics(client, archive_coverage_data):
+
# check view gets rendered without errors
url = reverse("swh-coverage")
resp = check_html_get_response(
@@ -117,8 +119,11 @@
logo_url = f'{settings.STATIC_URL}img/logos/{origins["type"].lower()}.png'
assert_contains(resp, f'src="{logo_url}"')
+ origin_visit_types = set()
+
if "instances" in origins:
for visit_types_ in origins["instances"].values():
+ origin_visit_types.update(visit_types_.keys())
for data in visit_types_.values():
if data["count"]:
assert_contains(resp, f'<a href="{escape(data["search_url"])}"')
@@ -126,7 +131,7 @@
for search_url in origins["search_urls"].values():
assert_contains(resp, f'<a href="{escape(search_url)}"')
- for visit_type in visit_types:
+ for visit_type in origin_visit_types:
assert_contains(resp, f"<td>{visit_type}</td>")
# check request as in production with cache enabled
@@ -135,7 +140,7 @@
)
-def test_coverage_view_with_focus(client):
+def test_coverage_view_with_focus(client, archive_coverage_data):
origins = (
listed_origins["origins"]
@@ -164,3 +169,93 @@
'class="collapse show"',
count=len(origins),
)
+
+
+@pytest.fixture
+def archive_coverage_data_with_non_visited_origins(mocker, swh_scheduler):
+ # mock calls to get nixguix origin counts
+ mock_archive = mocker.patch("swh.web.misc.coverage.archive")
+ mock_archive.lookup_latest_origin_snapshot.return_value = {"id": "some-snapshot"}
+ mock_archive.lookup_snapshot_sizes.return_value = {"release": 30095}
+
+ listers = []
+ for i, origins in enumerate(listed_origins["origins"]):
+ # create one instances for each lister
+ lister = swh_scheduler.get_or_create_lister(
+ origins["type"], f"instance-{origins['type']}"
+ )
+ listers.append(lister)
+
+ if i % 2 == 1 or origins["type"] in ("guix", "nixos"):
+ # do not declare origins for lister with odd index
+ continue
+
+ _origins = []
+ origin_visit_stats = []
+ for j, visit_type in enumerate(visit_types):
+ url = str(uuid.uuid4())
+ _origins.append(
+ ListedOrigin(
+ lister_id=lister.id,
+ url=url,
+ visit_type=visit_type,
+ extra_loader_arguments={},
+ )
+ )
+ # do not declare visit for visit type with even index
+ if j % 2 != 0:
+ now = datetime.now(tz=timezone.utc)
+ origin_visit_stats.append(
+ OriginVisitStats(
+ url=url,
+ visit_type=visit_type,
+ last_successful=now,
+ last_visit=now,
+ last_visit_status=LastVisitStatus.successful,
+ last_snapshot=os.urandom(20),
+ )
+ )
+ # send origins data to scheduler
+ swh_scheduler.record_listed_origins(_origins)
+ swh_scheduler.origin_visit_stats_upsert(origin_visit_stats)
+
+ # compute scheduler metrics
+ swh_scheduler.update_metrics()
+
+ # set deposit origins as empty
+ get_deposits_list = mocker.patch("swh.web.misc.coverage.get_deposits_list")
+ get_deposits_list.return_value = []
+
+
+def test_coverage_view_filter_out_non_visited_origins(
+ client, archive_coverage_data_with_non_visited_origins
+):
+
+ origins = copy.copy(listed_origins)
+
+ # check view gets rendered without errors
+ url = reverse("swh-coverage")
+ resp = check_html_get_response(
+ client,
+ url,
+ status_code=200,
+ template_used="misc/coverage.html",
+ server_name=SWH_WEB_SERVER_NAME,
+ )
+
+ for i, origins in enumerate(origins["origins"]):
+ if origins["type"] in ("guix", "nixos"):
+ continue
+ if i % 2 == 1:
+ # counters for lister with odd index should not be displayed
+ assert_not_contains(resp, f'id="{origins["type"]}"')
+ else:
+ # counters for lister with even index should be displayed
+ assert_contains(resp, f'id="{origins["type"]}"')
+ for j, visit_type in enumerate(visit_types):
+ if j % 2 == 0:
+ # counter for visit type with even index should be displayed
+ assert_not_contains(resp, f'id="{origins["type"]}-{visit_type}"')
+ else:
+ # counter for visit type with odd index should not be displayed
+ assert_contains(resp, f'id="{origins["type"]}-{visit_type}"')
diff --git a/swh/web/tests/utils.py b/swh/web/tests/utils.py
--- a/swh/web/tests/utils.py
+++ b/swh/web/tests/utils.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 The Software Heritage developers
+# Copyright (C) 2020-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -195,7 +195,12 @@
def check_html_get_response(
- client: Client, url: str, status_code: int, template_used: Optional[str] = None
+ client: Client,
+ url: str,
+ status_code: int,
+ template_used: Optional[str] = None,
+ http_origin: Optional[str] = None,
+ server_name: Optional[str] = None,
) -> HttpResponse:
"""Helper function to check HTML responses for a GET request.
@@ -209,7 +214,12 @@
The HTML response
"""
response = check_http_get_response(
- client, url, status_code, content_type="text/html"
+ client,
+ url,
+ status_code,
+ content_type="text/html",
+ http_origin=http_origin,
+ server_name=server_name,
)
if template_used is not None:
assert_template_used(response, template_used)